{
 "cells": [
  {
   "cell_type": "raw",
   "id": "c6227d68-b1f4-4f71-9cc6-18aa3ce54209",
   "metadata": {},
   "source": [
    "# First‑Page URL Summarizer (OpenAI)\n",
    "\n",
    "#This notebook does not crawl a whole site. It only fetches the first page for each provided URL and asks OpenAI to summarize it.\n",
    "\n",
    "### What it does\n",
    "Loads a list of URLs (provided inline or from a file)\n",
    "Fetches each page with `aiohttp` (HTML only)\n",
    "Extracts text via BeautifulSoup (basic)\n",
    "Calls OpenAI to produce a structured JSON summary\n",
    "Exports a CSV with: url, http_status, title, meta_description, summary, category, key_entities\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b0fe0e9-228e-461b-9a3e-f4392974c974",
   "metadata": {},
   "outputs": [],
   "source": [
    "# (Optional) If running locally, install deps here\n",
    "import sys, subprocess\n",
    "def pip_install(pkgs):\n",
    "    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", *pkgs])\n",
    "\n",
    "pkgs = [\n",
    "    \"aiohttp>=3.10\",\n",
    "    \"beautifulsoup4>=4.12\",\n",
    "    \"lxml>=5.2\",\n",
    "    \"pandas>=2.2\",\n",
    "    \"python-dotenv>=1.0\",\n",
    "    \"openai>=1.51\",\n",
    "]\n",
    "try:\n",
    "    import aiohttp, bs4, lxml, pandas, dotenv, openai\n",
    "except Exception:\n",
    "    pip_install(pkgs)\n",
    "print(\"Ready ✔\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86134741-0f8c-4049-894c-f31b27701da8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, asyncio, aiohttp, pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urlparse\n",
    "from dotenv import load_dotenv\n",
    "from openai import OpenAI\n",
    "\n",
    "load_dotenv()  # reads .env if present\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "MODEL = os.getenv(\"OPENAI_DEFAULT_MODEL\", \"gpt-4.1-mini\")\n",
    "if not OPENAI_API_KEY:\n",
    "    print(\"Set OPENAI_API_KEY in .env or environment.\")\n",
    "client = OpenAI(api_key=OPENAI_API_KEY)\n",
    "\n",
    "DEFAULT_HEADERS = {\"User-Agent\": \"FirstPageSummarizer/1.0 (+https://edwarddonner.com\"}"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b96c4ed0-4c50-4347-8cc4-22ea21e7e483",
   "metadata": {},
   "source": [
    "## 1) Provide URLs\n",
    "You can paste a small list below, or set `URLS_FILE` to a text/CSV file containing URLs (one per line or in a column named `url`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ce4aef5-8df8-4f47-91b3-c3ecc7c4c8be",
   "metadata": {},
   "outputs": [],
   "source": [
    "URLS_INLINE = [\n",
    "    \"https://edwarddonner.com\"\n",
    "]\n",
    "URLS_FILE = None  # e.g., \"urls.txt\" or \"urls.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba9f6f25-a04c-44fe-a16c-f7b5c47ed100",
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "def load_urls(urls_inline, urls_file):\n",
    "    urls = []\n",
    "    if urls_file and os.path.exists(urls_file):\n",
    "        if urls_file.endswith(\".csv\"):\n",
    "            df = pd.read_csv(urls_file)\n",
    "            if \"url\" in df.columns:\n",
    "                urls.extend(df[\"url\"].dropna().tolist())\n",
    "        else:\n",
    "            with open(urls_file, \"r\", encoding=\"utf-8\") as f:\n",
    "                for line in f:\n",
    "                    line=line.strip()\n",
    "                    if line:\n",
    "                        urls.append(line)\n",
    "    urls.extend([u for u in urls_inline if u])\n",
    "    # de-dup while preserving order\n",
    "    seen=set(); out=[]\n",
    "    for u in urls:\n",
    "        if u not in seen:\n",
    "            seen.add(u); out.append(u)\n",
    "    return out\n",
    "\n",
    "URLS = load_urls(URLS_INLINE, URLS_FILE)\n",
    "print(f\"Loaded {len(URLS)} URLs\")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "bb3761f0-3684-4f30-92e9-869fd4556529",
   "metadata": {},
   "source": [
    "## 2) Fetch first page HTML only\n",
    "This grabs the main HTML and extracts simple metadata and body text. No link following."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a7582b6-8277-4967-9d98-8cceeeab486d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from aiohttp import ClientTimeout\n",
    "from bs4 import BeautifulSoup\n",
    "try:\n",
    "    from bs4 import FeatureNotFound\n",
    "except Exception:\n",
    "    class FeatureNotFound(Exception):\n",
    "        ...\n",
    "\n",
    "DEFAULT_HEADERS = {\"User-Agent\": \"FirstPageSummarizer/1.0 (+https://edwarddonner.com)\"}\n",
    "\n",
    "async def fetch_one(session, url):\n",
    "    \"\"\"Fetch just one page (HTML if available).\"\"\"\n",
    "    try:\n",
    "        async with session.get(\n",
    "            url,\n",
    "            timeout=ClientTimeout(total=20),\n",
    "            headers=DEFAULT_HEADERS,\n",
    "            allow_redirects=True\n",
    "        ) as r:\n",
    "            ctype = r.headers.get(\"Content-Type\", \"\") or \"\"\n",
    "            is_html = \"html\" in ctype.lower()\n",
    "            text = await r.text(errors=\"ignore\") if is_html else \"\"\n",
    "            return {\n",
    "                \"url\": str(r.url),\n",
    "                \"status\": r.status,\n",
    "                \"content_type\": ctype,\n",
    "                \"html\": text,\n",
    "            }\n",
    "    except Exception as e:\n",
    "        return {\"url\": url, \"status\": None, \"content_type\": \"\", \"html\": \"\", \"error\": str(e)}\n",
    "\n",
    "def make_soup(html: str) -> BeautifulSoup:\n",
    "    \"\"\"Try lxml parser first, fall back to built-in html.parser if missing.\"\"\"\n",
    "    try:\n",
    "        return BeautifulSoup(html, \"lxml\")\n",
    "    except FeatureNotFound:\n",
    "        return BeautifulSoup(html, \"html.parser\")\n",
    "\n",
    "def extract_fields(url, html):\n",
    "    \"\"\"Extract title, meta description, and text from HTML.\"\"\"\n",
    "    soup = make_soup(html)\n",
    "    title = soup.title.string.strip() if soup.title and soup.title.string else \"\"\n",
    "\n",
    "    meta_desc = \"\"\n",
    "    m = soup.find(\"meta\", attrs={\"name\": \"description\"})\n",
    "    if m and m.get(\"content\"):\n",
    "        meta_desc = m[\"content\"].strip()\n",
    "\n",
    "    for tag in soup([\"script\", \"style\", \"noscript\"]):\n",
    "        tag.decompose()\n",
    "\n",
    "    text = soup.get_text(\" \", strip=True)\n",
    "    text = text[:8000]  # truncate to limit token size\n",
    "    return title, meta_desc, text\n",
    "\n",
    "async def fetch_all(urls):\n",
    "    \"\"\"Fetch and extract fields for a list of URLs (first page only).\"\"\"\n",
    "    import aiohttp\n",
    "    out = []\n",
    "    async with aiohttp.ClientSession() as session:\n",
    "        for u in urls:\n",
    "            resp = await fetch_one(session, u)\n",
    "            if resp.get(\"html\"):\n",
    "                title, meta_desc, text = extract_fields(resp[\"url\"], resp[\"html\"])\n",
    "                resp.update({\"title\": title, \"meta_description\": meta_desc, \"text\": text})\n",
    "            out.append(resp)\n",
    "    return out\n",
    "\n",
    "# Example usage in notebook (if URLS is defined):\n",
    "# results = await fetch_all(URLS)\n",
    "# len(results), results[:1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d30a3c6d-b208-4d6b-a5ea-e4276935a629",
   "metadata": {},
   "outputs": [],
   "source": [
    "URLS = [\"https://edwarddonner.com\", \"https://www.wikipedia.org/\"]\n",
    "results = await fetch_all(URLS)\n",
    "len(results), results[:1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2a53f08-4374-4125-9de8-6e1060e31200",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, json\n",
    "from dotenv import load_dotenv\n",
    "from openai import OpenAI\n",
    "\n",
    "load_dotenv()\n",
    "client = OpenAI(api_key=os.getenv(\"OPENAI_API_KEY\"))\n",
    "MODEL = os.getenv(\"OPENAI_DEFAULT_MODEL\", \"gpt-4.1-mini\")\n",
    "\n",
    "SYSTEM_PROMPT = \"\"\"\n",
    "You summarize a web page for migration planning. \n",
    "Return JSON with:\n",
    "- title: short page title\n",
    "- meta_description: concise (<= 160 chars)\n",
    "- summary: 3-5 bullet points as a single string\n",
    "- category: one of [blog, docs, product, pricing, careers, marketing, legal, support, account, other]\n",
    "- key_entities: array of 3-8 important entities/keywords\n",
    "\"\"\"\n",
    "\n",
    "def summarize_page(row):\n",
    "    user = (\n",
    "        f\"URL: {row['url']}\\n\"\n",
    "        f\"<title>{row.get('title','')}</title>\\n\"\n",
    "        f\"<meta_description>{row.get('meta_description','')}</meta_description>\\n\"\n",
    "        f\"<text>\\n{row.get('text','')[:6000]}\\n</text>\"\n",
    "    )\n",
    "    resp = client.responses.create(\n",
    "        model=MODEL,\n",
    "        input=[\n",
    "            {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n",
    "            {\"role\": \"user\", \"content\": user},\n",
    "        ],\n",
    "        response_format={\"type\": \"json_object\"}\n",
    "    )\n",
    "    return json.loads(resp.output[0].content[0].text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59f7d992-e7f0-4287-bd19-f8062fefe8c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "enriched = []\n",
    "for r in results:\n",
    "    if r.get(\"status\") and 200 <= r[\"status\"] < 400 and \"html\" in r.get(\"content_type\",\"\").lower():\n",
    "        try:\n",
    "            data = summarize_page(r)\n",
    "            enriched.append({**r, **data})\n",
    "        except Exception as e:\n",
    "            enriched.append({**r, \"error\": str(e)})\n",
    "    else:\n",
    "        enriched.append({**r, \"error\": \"Non-HTML or bad status\"})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "822d8108-64c2-4cf1-abc5-1acd288b7574",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(enriched)\n",
    "df.to_csv(\"firstpage_summary.csv\", index=False)\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f05d05c-bf6d-4236-8767-8695e4d4618f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
